{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Rental Listing Inquiries 数据集是 Kaggle 平台上的一个分类竞赛任务，需要根据\n",
    "公寓的特征来预测其受欢迎程度（用户感兴趣程度分为高、中、低三类）。其\n",
    "中房屋的特征 x 共有 14 维，响应值 y 为用户对该公寓的感兴趣程度。评价标准\n",
    "为 logloss。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "调用必要的模块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#导入xgboost分类器工具包\n",
    "from xgboost import XGBClassifier    #调用sklearn框架下的xgb分类学习器\n",
    "import xgboost as xgb                #直接调用xgboost（确定弱学习器数目的时候使用）\n",
    "\n",
    "#导入数据处理工具包\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#导入交叉验证工具包\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "#导入评价指标工具包\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "#导入可视化工具包\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "读取数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>40.7145</td>\n",
       "      <td>-73.9425</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>40.7947</td>\n",
       "      <td>-73.9667</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7388</td>\n",
       "      <td>-74.0018</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>40.7539</td>\n",
       "      <td>-73.9677</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>40.8241</td>\n",
       "      <td>-73.9493</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 225 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  latitude  longitude  price  price_bathrooms  \\\n",
       "0        1.5         3   40.7145   -73.9425   3000           1200.0   \n",
       "1        1.0         2   40.7947   -73.9667   5465           2732.5   \n",
       "2        1.0         1   40.7388   -74.0018   2850           1425.0   \n",
       "3        1.0         1   40.7539   -73.9677   3275           1637.5   \n",
       "4        1.0         4   40.8241   -73.9493   3350           1675.0   \n",
       "\n",
       "   price_bedrooms  room_diff  room_num  Year       ...        walk  walls  \\\n",
       "0      750.000000       -1.5       4.5  2016       ...           0      0   \n",
       "1     1821.666667       -1.0       3.0  2016       ...           0      0   \n",
       "2     1425.000000        0.0       2.0  2016       ...           0      0   \n",
       "3     1637.500000        0.0       2.0  2016       ...           0      0   \n",
       "4      670.000000       -3.0       5.0  2016       ...           0      0   \n",
       "\n",
       "   war  washer  water  wheelchair  wifi  windows  work  interest_level  \n",
       "0    0       0      0           0     0        0     0               1  \n",
       "1    0       0      0           0     0        0     0               2  \n",
       "2    0       0      0           0     0        0     0               0  \n",
       "3    0       0      0           0     0        0     0               2  \n",
       "4    1       0      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 225 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"RentListingInquries_FE_train.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 225 entries, bathrooms to interest_level\n",
      "dtypes: float64(7), int64(218)\n",
      "memory usage: 84.7 MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "准备数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train=train[\"interest_level\"]\n",
    "x_train=train.drop(['interest_level'],axis=1)\n",
    "x_train=np.array(x_train)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#y_train = train['interest_level']\n",
    "#train = train.drop( 'interest_level', axis=1)\n",
    "#X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#使用xgboost自带的cv对参数n_cstimators进行调优：需要设置的数值太多，不适合用GridSearchCV\n",
    "def modlefit(alg,x_train,y_train,cv_folds=None,early_stopping_rounds=10):\n",
    "    xgb_param=alg.get_xgb_params()   #获取alg模块中的参数\n",
    "    xgb_param['num_class']=3         #3类分类问题\n",
    "    \n",
    "    #直接调用xgboost，而非sklearn中的wrapper类\n",
    "    xgtrain =xgb.DMatrix(x_train, label=y_train)\n",
    "    \n",
    "    cvresult = xgb.cv(xgb_param,xgtrain,num_boost_round=alg.get_params()['n_estimators'],folds=cv_folds,\n",
    "            metrics='mlogloss',early_stopping_rounds=early_stopping_rounds)\n",
    "    \n",
    "    cvresult.to_csv('1_nestimators.csv',index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators=cvresult.shape[0]\n",
    "    \n",
    "    #采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators=n_estimators)\n",
    "    alg.fit(x_train,y_train,eval_metric='mlogloss')\n",
    "kfold = StratifiedKFold(n_splits=5,shuffle=True,random_state=3)\n",
    "#先构造xgbclassifier分类器，参数learning_rate先设置为0.1，其余参数设置在合理范围内即可\n",
    "xgb1=XGBClassifier(\n",
    "     learning_rate=0.1,\n",
    "     n_estimators=1000,\n",
    "     max_depth=5,\n",
    "     min_child_weight=1,\n",
    "     gamma=0,\n",
    "     subsample=0.7,\n",
    "     colsample_bytree=0.8,\n",
    "     colsample_bylevel=0.7,\n",
    "     objective=\"multi:softprob\",\n",
    "     seed=3)\n",
    "modlefit(xgb1,x_train,y_train,cv_folds=kfold,early_stopping_rounds=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'base_score': 0.5,\n",
       " 'booster': 'gbtree',\n",
       " 'colsample_bylevel': 0.7,\n",
       " 'colsample_bytree': 0.8,\n",
       " 'gamma': 0,\n",
       " 'learning_rate': 0.1,\n",
       " 'max_delta_step': 0,\n",
       " 'max_depth': 5,\n",
       " 'min_child_weight': 1,\n",
       " 'missing': None,\n",
       " 'n_estimators': 446,\n",
       " 'nthread': 1,\n",
       " 'objective': 'multi:softprob',\n",
       " 'reg_alpha': 0,\n",
       " 'reg_lambda': 1,\n",
       " 'scale_pos_weight': 1,\n",
       " 'seed': 3,\n",
       " 'silent': 1,\n",
       " 'subsample': 0.7}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb1.get_xgb_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\anaconda\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3XmcXXV9//HXZ+6dNTOTyTJZJxsQloCAGDZRiCA1UAVFRINYsVaqlWq11oL1R5GWqm1dsFLUUqGWCkVURKRii4CIgAk7SViyQSYJWSfJZJn98/vje+7MmcmdmTvJ3Dl35r6fj8d5zD3nfM8533tm5r7v93s2c3dEREQASpKugIiIFA6FgoiIdFMoiIhIN4WCiIh0UyiIiEg3hYKIiHRTKIjEmNkXzOzmpOshkhSFwihjZtVmts7MLo1NqzGz18zs4ti0hWZ2r5k1mdlOM1thZteb2YRo/uVm1mlme6JhjZl9Is91X2RmjfncxlBkq4+7/4O7/0metrfOzN6ej3Xnw0j9vkbbfhnrFAqjjLvvAa4AbjCz+mjyPwLL3P0uADN7M/AQ8ChwtLvXAYuBDuCE2Ooec/dqd68GLgb+0czeODLvRIbCzNJJ10GKhLtrGIUDcCtwO7AI2A5Mj837LfAvgyx/OfDbPtN+D1waG78AWA7sJITMMbF5x0TTdkZlLojNOx9YATQDG4DPAeOA/UAXsCcaZvTzvm4EfhEt/wRweA7742jgf4EdwEvAJQdTH+Ba4LZoubmAAx8B1gNNwMeBk4Hnovf+7dh2Dgd+Hf0+tgH/BdRF8/4z2tb+aFufz2EfrwP+OtpWK5COxjdE7+Ul4Jws++I04HUgFZv2HuC56PUpwDJgN7AZ+Ho/+3QR0NjPvPHAD4CtwKvAF4GSaF4K+Fq0D9YCV0b7Md3PutYBb+9n3seAVdHv9Z7M3wxgwDeALcCuaB8d19/vO+n/19E0JF4BDQf5i4MJwKboH+8jsenjgE5g0SDLX04sFKIPup3AkdH4kcBe4FygFPh89M9ZFo2vAr4QjZ8d/QMeFS27CXhrrJ4nRa/7/ZCJ1ePW6APglOhD8L+AOwZZZhzhQ/sj0TInRfvl2KHWh+yh8B2gAvgDoAW4G5gCzIw+lM6Kyh8R7a9yoB74DfDN2Lp7ffgNtI9j5Z8BZgGVwFHR+5wRq1/WwARWA+fGxn8EXBW9fgz4UPS6Gjitn3X0+/siBMLPgJqoHi8DH43mfZzwodwQ7e//4yBCIfq72hb9PsuBfwF+E817B/AkUEcIiGOIvhj19/vWkNug7qNRyt2bCN8wq4CfxGZNIHQLvp6ZYGb/GB1X2GtmX4yVPS2avofQSvhP4JVo3vuBX7j7/7p7O/DPhA+mNxO+iVYDX3H3Nnf/NXAvsCRath1YYGa17t7k7k8N8e39xN1/7+4dhFA4cZDy7wTWufst7t4Rbe/HhC6x4ajP37l7i7v/ivAhfru7b3H3DcAjwBsB3H1VtL9a3X0r8HXgrAHWO9A+zviWu6939/2EsC+P3kupu69z99X9rPt2ot+HmdUQvj3fHtsfR5jZZHff4+6PD2VnmFkqqvvV7t7s7usILYMPRUUuAW5w98bo7/QrQ1l/zAeB77v7U+7eClwNnG5mc6P3UENoIZq7r3T3TbH3dyi/76KmUBilzOwywje0/wO+GpvVROimmJ6Z4O6f93Bc4aeEb9IZj7t7nYdjCtOAY4F/iObNIHQLZNbRRfiWOjOatz6alvFqNA/gvYQPoVfN7GEzO32Ib+/12Ot9hAAayBzg1CjgdprZTsIHyrRhqs/m2Ov9WcarAcxsipndYWYbzGw3cBsweYD1DrSPM9bH5q8C/oLQmtkSbWtGP+v+IXCRmZUDFwFPuXtmWx8ltFJeNLOlZvbOAeqYzWRCC/HV2LT4739GvN59Xg9F3/2zh9A1NzP6IvJtQlfjZjP7npnVRkUP9fdd1BQKo5CZTSH0p34M+FPgEjM7E8Dd9xL64S8ayjrdfTPh2/W7okkbCR+2mW0aoRtjQzRvlpnF/35mR/Nw96XufiGhi+Vu4M7MZoZSpyFYDzwcBVxmqHb3T4xwfb4crfN4d68FLiN0bWT03d5A+zjrMu7+Q3d/S7Sc0/sLQbzcCsIH6nnApYSQyMx7xd2XEPbHV4G7zGxc7m+TbYRv43Ni07p//4Tum4bYvFlDWHdc3/0zDphEz9/Zt9z9TYQvM0cCfxVN7+/3LTlQKIxO3wbudvcHoybz54F/i74VEo3/sZldFQUIZtYAzOtvhWY2iXAwcnk06U7gD83sHDMrBf6ScLDzd4TQ2Qt83sxKzWwRIUzuMLMyM/ugmY2PukR2E7o9IHzDnmRm44dpP2TcCxxpZh+K6lNqZieb2TEjXJ8awkHknWY2k+hDKmYzcFhsfKB9fAAzO8rMzo5+zy2EVkpntrKRHwKfAs4kHFPIrOcyM6uPWiY7o8n9rsfMKuIDoSV6J3B9dDr0HOCzhJZR5n192sxmmlkd4eD4YEr7bCcd1f8jZnZi9J7/AXjC3ddFv99To/22N9ofnYP8viUXSR/U0DC0AXg34RtUXZ/pDwDXx8ZPBe4j/NPvBF4ArgcmRfMvJ/yzZM682ULoc54SW8d7CAcMdwEPEx24jeYdG03bFZV5TzS9DPgloRtrN7AUeEtsue8TugB20v/ZR38fG1/EIAeno3JHEc5Y2hqt/9eEYxFDqg/ZDzSnY+UbiR3EJ3wQfjG2T56M9uczhA/5xljZC4HXom19Lod9vI7eB6aPJxz7aSYcjL832z6MlZ9N+AD/RZ/pt0W/7z2ELwHv7mf5RdH77zscQTh2dVu0v9cD19Bz9lGa0JLdTjj76DOEloX1s511Wbbx99G8jxMOmmfeb0M0/RzCGUd76DnTq3qw37eGwQeLdrCISF6Y2XnAd9x9zqCFJXHqPhKRYWVmlWZ2vpmlo260vyWc5CCjgFoKMiqY2VuB/8k2z8PZU1IgzKyK0BV2NOG4xy+AT7v77kQrJjlRKIiISDd1H4mISLdRd5OtyZMn+9y5c5OuhojIqPLkk09uc/f6wcqNulCYO3cuy5YtS7oaIiKjipm9OngpdR+JiEiMQkFERLopFEREpJtCQUREuikURESkm0JBRES6KRRERKRb0YTCpu07WfrCi3hX1+CFRUSKVNGEws9u+htOvutU9u/fm3RVREQKVtGEwklHhKcD7t/bnHBNREQKV9GEgpWHuyu3KBRERPpVNKGQrgjPJW/dr1u6i4j0p2hCIVURWgqt+/YkXBMRkcJVNKFQVhlCoWO/uo9ERPpTNKGQjloKHS1qKYiI9KdoQqFiXC0AHS06JVVEpD9FEwrlVTUAdLWqpSAi0p+iCYWK7lBQS0FEpD9FEwpV1eMB8DaFgohIf/IWCmb2fTPbYmYv9DPfzOxbZrbKzJ4zs5PyVReAVGk57Z6CdoWCiEh/8tlSuBVYPMD884D50XAFcFMe6wJAi5Vj7fvyvRkRkVErb6Hg7r8BdgxQ5ELgBx48DtSZ2fR81QdgP5WUqKUgItKvJI8pzATWx8Ybo2kHMLMrzGyZmS3bunXrQW+wtaSCVMf+g15eRGSsSzIULMs0z1bQ3b/n7gvdfWF9ff1Bb7DNFAoiIgNJMhQagVmx8QZgYz432JaqpLRLxxRERPqTZCjcA/xRdBbSacAud9+Uzw12tLVR2rozn5sQERnV0vlasZndDiwCJptZI/C3QCmAu38HuA84H1gF7AM+kq+6ZHRUT6Nm39p8b0ZEZNTKWyi4+5JB5jvwyXxtP5vOdBXlXTqmICLSn6K5ohnAS6uooDXpaoiIFKyiC4VKbyE0UkREpK8iC4VxVFg7be3tSVdFRKQgFVUoWFl4TvO+PXr6mohINkUVCqnyEAot+3YnXBMRkcJUVKFg5eGRnK17FQoiItkUVSikK0NLoXW/nr4mIpJNcYVCeXj6Wvt+HVMQEcmmqEKhrCp0H7WrpSAiklVxhUJlaCl0tCgURESyKa5QqAqh0NWqUBARyaaoQqFqXC0AHS/en3BNREQKU1GFwrjaiQD4tOMSromISGEqqlBIlVXS7imsRWcfiYhkU1ShgBl7bBzWpovXRESyKa5QAPaXjCPdrpaCiEg2RRgK1ZQpFEREsiq6UGhNV1PWqVNSRUSyKbpQaE9XU9G5N+lqiIgUpKILhY6yWqpcoSAikk3RhYKX1VDte/VIThGRLIouFKiopdpa2N/alnRNREQKTtGFglWMB2DP7qaEayIiUniKLhRKquoA2LtrR8I1EREpPHkNBTNbbGYvmdkqM7sqy/w5ZvaAmT1nZg+ZWUM+6wNQWhVaCvv3qKUgItJX3kLBzFLAjcB5wAJgiZkt6FPsn4EfuPvxwHXAl/NVn4yycRMAaN2jloKISF/5bCmcAqxy9zXu3gbcAVzYp8wC4IHo9YNZ5g+7yprQfdS2d1e+NyUiMurkMxRmAutj443RtLhngfdGr98D1JjZpL4rMrMrzGyZmS3bunXrIVWqsiasvnPfzkNaj4jIWJTPULAs0/peHPA54Cwzexo4C9gAdBywkPv33H2huy+sr68/pEplnqnQuU8tBRGRvtJ5XHcjMCs23gBsjBdw943ARQBmVg28193z+mldEXUf0aJQEBHpK58thaXAfDObZ2ZlwAeAe+IFzGyymWXqcDXw/TzWJ2wzXU4LpdCqZyqIiPSVt1Bw9w7gSuB+YCVwp7svN7PrzOyCqNgi4CUzexmYClyfr/rE7bVqUm26fbaISF/57D7C3e8D7usz7ZrY67uAu/JZh2z2mR60IyKSTdFd0Qyws6uC9r06+0hEpK+iDIWUORNRKIiI9FWUodBedxjjrCXpaoiIFJyiDIXOionUeTOdXXqmgohIXFGGAlWTqLV97GzWE9hEROKKMhRS1ZMB2N20OeGaiIgUlqIMhbKacKuM5u0KBRGRuKIMhYq6EAotu7YkXBMRkcJSlKFQM2EqAK27tyVcExGRwlKUoVAdhULHnkO7DbeIyFhTlKFQURvdfnvf9mQrIiJSYIoyFEiXsYcqSvbrkZwiInHFGQpAc0ktpa1NSVdDRKSgFG0o7EvXUdGuUBARiSvaUGgtq6OqQ09fExGJK9pQ6CifQE2Xnr4mIhJXtKHQVTmRCTSzv60z6aqIiBSMog2FpVuMKmtlx04dVxARySjaUHjL8UcB0LTt9YRrIiJSOIo2FKomTgdg97aNCddERKRwFG0o1NbPBqC1aUPCNRERKRxFGwrj6xsAaN+5KeGaiIgUjqINhZKaqXRh0KxjCiIiGUUbCqTS7Cypo2y/HrQjIpKR11Aws8Vm9pKZrTKzq7LMn21mD5rZ02b2nJmdn8/69LUnPYnKVj1TQUQkI2+hYGYp4EbgPGABsMTMFvQp9kXgTnd/I/AB4F/zVZ9s9ldMYXyHQkFEJGPQUDCzw82sPHq9yMw+ZWZ1Oaz7FGCVu69x9zbgDuDCPmUcqI1ejwdG9PzQjnFTmeQ7aGnXVc0iIpBbS+HHQKeZHQH8OzAP+GEOy80E1sfGG6NpcdcCl5lZI3Af8OfZVmRmV5jZMjNbtnXrMD4trXoqk2hmc1Pz8K1TRGQUyyUUuty9A3gP8E13/wwwPYflLMs07zO+BLjV3RuA84H/NLMD6uTu33P3he6+sL6+PodN56asbiYl5uzYomsVREQgt1BoN7MlwIeBe6NppTks1wjMio03cGD30EeBOwHc/TGgApicw7qHRdXkcK1C87bGkdqkiEhByyUUPgKcDlzv7mvNbB5wWw7LLQXmm9k8MysjHEi+p0+Z14BzAMzsGEIoDGP/0MDGR6HQskOhICICOYSCu69w90+5++1mNgGocfev5LBcB3AlcD+wknCW0XIzu87MLoiK/SXwMTN7FrgduNzd+3Yx5c246KrmZ5avHKlNiogUtPRgBczsIeCCqOwzwFYze9jdPzvYsu5+H+EAcnzaNbHXK4AzhljnYWPjptDpxrFdLydVBRGRgpJL99F4d98NXATc4u5vAt6e32qNkFSanel6KtPZjomLiBSfXEIhbWbTgUvoOdA8ZjRXzmR82yZGsNdKRKRg5RIK1xGOC6x296VmdhjwSn6rNXLaa2cxky1s39uWdFVERBKXy4HmH7n78e7+iWh8jbu/N/9VGxm/217NNGuiccuOpKsiIpK4XG5z0WBmPzWzLWa22cx+bGYNI1G5kfCOM04BYPuG1QnXREQkebl0H91CuL5gBuE2FT+Ppo0JE164FYC9mxUKIiK5hEK9u9/i7h3RcCswfPeaSFj5ZbcD0LF9XbIVEREpALmEwjYzu8zMUtFwGbA93xUbMdXTaKOU9O7Xkq6JiEjicgmFPyacjvo6sAm4mHDri7GhpISN1FPavH7wsiIiY1wuZx+95u4XuHu9u09x93cTLmQbM2zCHGayhT2tHUlXRUQkUQf75LVBb3Exmti+7cy1zazarOcqiEhxO9hQGFP3hRh3yoeotX289trapKsiIpKogw2FMXVPiLo5bwDgVw//JuGaiIgkq9+7pJpZM9k//A2ozFuNEpCachQAp9aM2KMcREQKUr+h4O41I1mRRNVMp4Uyqra9kHRNREQSdbDdR2OLGTtr5jPNt+oMJBEpagqFSOfEIzmiZAPv/85jSVdFRCQxCoXIuIZjmWo7+dBJdUlXRUQkMQqFyPhZxwKw9eGbE66JiEhycnlGc7azkHYBy4C/dPc1+ajYSLNp4bRU0uXJVkREJEGDhgLwdWAj8EPC6agfAKYBLwHfBxblq3IjanwDTdQydc+L7GntoLo8l10jIjK25NJ9tNjdv+vuze6+292/B5zv7v8NTMhz/UaOGV3TT+QNJWt4YcOupGsjIpKIXEKhy8wuMbOSaLgkNm9MXdlcuWc9820DL6x7PemqiIgkIpdQ+CDwIWBLNHwIuMzMKoEr81i3EVd1/t+Tti5+88iDSVdFRCQRg3acRweS39XP7N8OtKyZLQZuAFLAze7+lT7zvwG8LRqtAqa4e3LnhM44EYBjfDWdXU6qZEzd909EZFCDthTMrMHMfmpmW8xss5n92MwaclguBdwInAcsAJaY2YJ4GXf/jLuf6O4nAv8C/OTg3sYwqZ1Ji1VwZOfLrNy0O9GqiIgkIZfuo1uAe4AZwEzg59G0wZwCrHL3Ne7eBtwBXDhA+SXA7TmsN3/M4PBzOLXkRT5x25OJVkVEJAm5hEK9u9/i7h3RcCtQn8NyM4H4My4bo2kHMLM5wDzg1/3Mv8LMlpnZsq1b83sn04ojzqTBtnHyhL153Y6ISCHKJRS2mdllZpaKhsuA7Tksl61Dvr+zlT4A3OXundlmuvv33H2huy+sr88ljw7B3DMASK97mJb2rNURERmzcgmFPwYuAV4HNgEXAx/JYblGYFZsvIFwEVw2HyDprqOMKcfSXjaeN5W8wqOrtiVdGxGRETVoKLj7a+5+gbvXu/sUd383cFEO614KzDezeWZWRvjgv6dvITM7inARXGHcnrSkhNTcN3NaaiVX/fi5pGsjIjKiDvaGeJ8drIC7dxCuY7gfWAnc6e7Lzew6M7sgVnQJcIe7F8yFcCWHLWKObaZmXyMdnV1JV0dEZMQcbCjkdAK/u9/n7ke6++Hufn007Rp3vydW5lp3v+og65EfR74DgEX2JO/8lwEvxRARGVMONhQK5lt9XkycR1e6inNLlrFtT2vStRERGTH9hoKZNZvZ7ixDM+GahTGt5NSPcXLqZdr37qS5pT3p6oiIjIh+Q8Hda9y9NstQ4+5j/77SR55Hmk7OtGe497lNSddGRGRE6Mlr/Zl1Cl4znXeXPs7f3buCAjoOLiKSNwqF/pSkMCthkT1FadsuFn/zkaRrJCKSdwqFgSy5nRRdXFD6BJt27U+6NiIieadQGMi046G0kj8tuYfdLR08tjqXu3uIiIxeCoWBmME519JgWzkpvZaP/sdSurp0bEFExi6FwmBOvBTKqvnmvN+zr62Tc77+cNI1EhHJG4XCYCpq4cRLmbXxf5huTazdtpfGpn1J10pEJC8UCrk47c8w7+IL4+8H4LwbHlE3koiMSQqFXEycBycs4V3t93PyxBaaWzo4658eTLpWIiLDTqGQqzM/B51t3GlXkS4x1jft56GXtiRdKxGRYaVQyNWEuVA7A9u7hRf+pI6qshSX37KU82/4TdI1ExEZNgqFofjk76F2JhW/+jwPfvYtmMGKTc0817gz6ZqJiAwLhcJQlFfDO/4BXn+eqStu5YSG8ZjBBd9+lPO+qRaDiIx+CoWhWnAhHHU+PPAl7r6oht/+9dlUlqZY+Xoz53ztoaRrJyJySBQKQxWaBlA1Ce76Y2ZWdvL41eeQMli9dS+nXP9/tLR3Jl1LEZGDolA4GOMmwUX/BttXwQ0nML4yzcq/O49ptRVsaW5lwTW/ZNm6HUnXUkRkyBQKB2veW6FuDuzbBg99mbJ0CY9/4RyOmlqNO1z8ncc44gv3sXl3S9I1FRHJmY22h8csXLjQly1blnQ1Anf45yNh7xZ4903hPknAvrYOTvzSr2jrDPu2LGUs/ZtzGV9VmmRtRaSImdmT7r5wsHJqKRwKM/jMcph3Fvzsk/DCTwCoKkvz8vXn85u/ehvpEqOt0znhul8x/2/UchCRwqaWwnBo2wtfOxpad8MlPwhnKMW881uPsHzjbjJ72oCjptVw36feSkmJjXh1RaT45NpSUCgMl5bdcNt7oXEpnPdVOPVPDyjy2vZ9nPP1h2jv7NnnBhzfMJ6fXfmWEaysiBSbgggFM1sM3ACkgJvd/StZylwCXAs48Ky7XzrQOgs2FCC0GL5xHOzfAadcAX9wPaTLDijW0t7JH37rEVZv3dtrelnKmD+1hp9f+Ra1IERkWCUeCmaWAl4GzgUagaXAEndfESszH7gTONvdm8xsirsPeJe5gg4FgK5O+NX/g8dvhLJq+MSj4b5J/Whs2sc5X3uY1o6uXtMNKEuXcPS0Gu7+5BmYKSRE5OAVQiicDlzr7u+Ixq8GcPcvx8r8I/Cyu9+c63oLPhQyVtwDP7o8vH7fLQccZ8jmon99lBWbdtPa3kXf30omJFIlxoLptdz1iTcPd41FZAwrhFC4GFjs7n8SjX8IONXdr4yVuZvQmjiD0MV0rbv/Msu6rgCuAJg9e/abXn311bzUedg1rYObzoC2PXDce2HxV6G6PqdF3Z1j//Z+9reFq6Oz/ZbibYdjZ9Ry58dPp6osfcjVFpGxpxBC4X3AO/qEwinu/uexMvcC7cAlQAPwCHCcu/d729FR01LI6GiD334DHvoylKTgXd8K1zMMsTvI3XnPjY/yXOMunOwhEZdZe0VpCUdOreEnf3YGKR2nEClahRAKuXQffQd43N1vjcYfAK5y96X9rXfUhULG1pfg5reH01bLa+HD98CMNx7yat930+9Yvmk3LW2d3UGRa2AYUFGWYtkX364WhsgYVwihkCZ0DZ0DbCAcaL7U3ZfHyiwmHHz+sJlNBp4GTnT37f2td9SGAkBXFzz9A/jF56CrHU64FM65BmqnD/umLvlOCIsuh5YBuqCyKU0ZHdFps5kAOX5WHXdccRoVpalhr6uI5F/ioRBV4nzgm4TjBd939+vN7DpgmbvfY+GUmq8Bi4FO4Hp3v2OgdY7qUMho2QWPfA0evSGMn/lXcNqfQdXEEdl8JjD2t3b2CoqD/UswQpC09wkSgMryFM9c8weUpnTxvEiSCiIU8mFMhELGjrXwwJdg+U/D+Jv/HE6/EmqmJVsv4L03PcrKTc39tjIO9a/GgHSsRRKfPpBxFWkWTK/lv//09EOsgUhxUSiMJltWwiNfh+fvDOMLPwqnfxImHZ5svYbgku/8jmXrmnpNy/aXla+/toHCpKK0hNb2rgFKHHjcv6pc4SNji0JhNNq+OnQpPfUfYfzws0NAHLkYUmPvQPD7v/sYS9cO/NyJvn+dmdZFvv9qzcJNcLPO62eZ0nQJ7VkuQsxFZXmK/a29H8402AlqVeVp9rV2DFhm4dyJCjYBFAqjW/Pr8NR/woPXAw6psnDc4aQ/KoiupULy/u8+dsADjfr+SVeUpYZ8sD1VYnR25T98DkWJQdcAFcyWKdm67AZTXlpC2yAtrQOWKUvR2ja0JxDGf0+5qixLdV/Lk6uq8hT7WvtfJlsY5xLA+V7mUFuvCoWxoLMDXv5luC13y07AwvOhj7sIjjoPysYlXcOi8v7vPsaKTbuz/tO6D/5hc8Aysddl6RLaOnL74M0slwkuKUzG0LtLBwr6EoPD6qv5v8+edXD1yTEUxl6fxFiSSsMx7wzD9tXw5C3wxHfhpV+AlcCxF8Gx7wndTGVVSdd2zCuUbphMOAGH9E001++DB/MNvjyH4zh9lWVpkQxWxfJ0yQH3DRt0O0MI4IyDaWEdzDIlZrhnb6GaGakRuP5ULYXRpqsTXv0dvHAXPH0bdHWEgDjqfDjmXXDkO6ByQtK1FJEBvOHa+7MGurqPDkLRh0JcZzus+y28eC889QPobAMMDlsUWhdHv1PHIEQEUCgUn64u2PgUrLwndDF1RI/9bDg5tB4OOxtmnBjuvyQiRUehUMzcYeuLsPLe0IrY9EyYXpKGo/8QDnsbHP62AZ/zICJjiw40FzMzmHJMGM76K9i7DdY8BKsfDBfIrfhZKDdhXgiHw94G886EyrpEqy0iyVNLodi4w7ZXYM2D8NBXwqNDM2Yu7AmJhpOzPkpUREYndR9JbjrboXFZCInH/xVam8N0K4Ejzu0JifqjhvwMCBEpHAoFOTgtu2DtIyEkVj8IO1aH6ePqQ+uhYWH4OeONUF6TbF1FJGc6piAHp2J8zwVzAE2vhuMRrz0OjUvhpfvCdCuBKQt6QqLhZJg0H0p0i2yR0UwtBRmafTtgw1MhIBqXwtqHwgV1AOXjoeFNPSEx800j9owIERmYWgqSH1UTYf7bwwDh+ojtq3pConEZPPzVnvKTjujd7TTl2DF5x1eRsUL/nXJoSkqg/sgwvPGDYVrrHtj4dE9IPH8XPHt7mFdaFY5HZEJi+gkwfpYOYosUCIWCDL/yapj31jBAOA1252s9IdG4FB771/CcagjdTlOPjQ3HhWssyquTew8iRUqhIPnBa+HNAAANDklEQVRnBhPmhOENF4dp7S3w+vPw+nOweXkYnr0D2pp7lpswryckph4L046Durk6mC2SRwoFSUZpBcw6OQwZmRZFJiQ2Px9+vnhvbLlxMHVB77CYskBXY4sME4WCFI54i+Lo83umt+2DrStjYbEclt8NT97aU2b8rN7dT1OPg4mH6aC2yBDpP0YKX1lVOL115pt6prlD86YoJF7oCYuX76f70SzpCqg/uqdFkQmMcZMSeRsio4FCQUYnM6idEYb55/ZM72iFrS/1DotXfgXP3NZTpnpa1O0U3TSw/phwGw8d2BZRKMgYky6H6ceHIW7Plt7dT5ufh9//Fjpbe8qMnx1OrZ18ZLi+YvL88LNmuk6ZlaKR11Aws8XADUAKuNndv9Jn/uXAPwEboknfdveb81knKVLVU8Jw+Nt6pnV1QtM62LIyHLPY8iJsezk87rR9X0+50nEw6fAoJKKgmHxE+Kn7P8kYk7dQMLMUcCNwLtAILDWze9x9RZ+i/+3uV+arHiL9KkmFD/tJh/fc6wnCVdrNG8OV2ttege2rYfsrsOFJeOHHvddRPa13SGRCY8IcSJWO7PsRGQb5bCmcAqxy9zUAZnYHcCHQNxRECktJCYxvCMNhi3rP62iFHWtDSMQDY+XPYd/22DrS4cl2k+YfGBjVU9QdJQUrn6EwE1gfG28ETs1S7r1mdibwMvAZd1+fpYxIYUiXw5Sjw9DXvh09IdHdylgFq3/d+9hFeW3UQunTFTXpCCgbN3LvRSSLfIZCtq9CfW/J+nPgdndvNbOPA/8BnH3AisyuAK4AmD179nDXU2R4VE0MQ/yCPAjdUbvWh4DIDNteCbcjf/7O3mVrZvSExIR54VqLifNCq0OBISMgb7fONrPTgWvd/R3R+NUA7v7lfsqngB3uPn6g9erW2TKmtO+PWherohbG6p4WRsvO3mWrp0UBEQ+LeeGnblEugyiEW2cvBeab2TzC2UUfAC6NFzCz6e6+KRq9AFiZx/qIFJ7SynBPp2nHHThvf1M4ftG0FnasgR3rwus1D8KzP+xdtmJ8CIp4YEw8LAzVU3UMQ3KWt1Bw9w4zuxK4n3BK6vfdfbmZXQcsc/d7gE+Z2QVAB7ADuDxf9REZdSonwMwJMPOkA+e17YOdr4bQ2LGmJzg2Pg3Lf9K7bLoyun3IXKiLfnYPc9QtJb3oyWsiY01ne7ixYNNa2L4mhEfTup6hbU/v8uPqswTGnNDqqJ0RTt2VUa8Quo9EJAmp0p7rL47oM889nCXVtC6ERjwwGpfC8p+Cd8bWVRbCInOwuzs45oTXFbUj9a5khCgURIqJWbgh4LhJ4XnafXW2w67GWMtibXRcY104W6p1d+/ylROioJjT+2fdHKibFY6ZyKiiUBCRHqnS6CD1vAPnuYeD3ztfhaZXe342rYPNK+ClX/a+HgPCQe7usJgdG+aEiwPT5SPytiR3CgURyY1Zz7UYM9544PyuLtizOYTFztei4FgXfq5/Ap7/0YHL1EwPz8LoFRjRUDsz3DZdRpRCQUSGR0kJ1E4Pw+zTDpzf2RGegbHztT7Dq7BhGay4G7o6ei9TNTm0KOpmhfAYPys2PjsElE63HVYKBREZGal0+DCvmwWcceD8rs4QGk2vhuMau9ZHQyNsfRlWPdD77rUQ7mCbCYy62dH6Z4fAqJsF46bomd5DpFAQkcJQkuq5EWE2mWMau9ZHLYz1sdevhdbG/qbey6TKo5bF7J7WRffrWTrlNguFgoiMDvFjGtNPyF6mtfnAsMi8fuZ26GrvXb4kHYKhbk7U2si0NKLXtQ2QLsv/eysgCgURGTvKa2DqgjBk074/dEf1CowoNNY+DLs3HLhM9TQYPzO0OGobwuvazPjMcCv0MdTaUCiISPEorQxP0Js8P/v8jrYQDL26qBphd2M47Xblz8G7DlyutqHnmeHxoSbzc/qoaXEoFEREMtJl/V+nAT3HNXZvgF0bQljs3hgNG8Lzv1fc3c/KLTw7vHZmT1DUzuxpedTOKIiL/RQKIiK5ih/XmPaG7GXcoWVXCIrmjbHQiIY1Dx14FlVGSRqmHBPrpprR0wrJhEeeL/hTKIiIDCczqKwLQ3/HNiDc6bZ5U9Q9tSHW+tgYuq9e/iUHPJdswmHw6afzWn2FgohIEsqqem5c2J+2vbB7U0831cxBb3J6yBQKIiKFqmxceDzr5L63u80fXeonIiLdFAoiItJNoSAiIt0UCiIi0k2hICIi3RQKIiLSTaEgIiLdFAoiItLN3H3wUgXEzLYCrx7k4pOBbcNYnbFA+6Q37Y/etD8ONFr3yRx3rx+s0KgLhUNhZsvcPf/XiY8i2ie9aX/0pv1xoLG+T9R9JCIi3RQKIiLSrdhC4XtJV6AAaZ/0pv3Rm/bHgcb0PimqYwoiIjKwYmspiIjIABQKIiLSrWhCwcwWm9lLZrbKzK5Kuj4jwcy+b2ZbzOyF2LSJZva/ZvZK9HNCNN3M7FvR/nnOzE5Krub5YWazzOxBM1tpZsvN7NPR9GLeJxVm9nszezbaJ1+Kps8zsyeiffLfZlYWTS+PxldF8+cmWf98MbOUmT1tZvdG40WzP4oiFMwsBdwInAcsAJaY2QAPTx0zbgUW95l2FfCAu88HHojGIeyb+dFwBXDTCNVxJHUAf+nuxwCnAZ+M/g6KeZ+0Ame7+wnAicBiMzsN+CrwjWifNAEfjcp/FGhy9yOAb0TlxqJPAytj48WzP9x9zA/A6cD9sfGrgauTrtcIvfe5wAux8ZeA6dHr6cBL0evvAkuylRurA/Az4Fztk+73VwU8BZxKuGI3HU3v/v8B7gdOj16no3KWdN2HeT80EL4cnA3cC1gx7Y+iaCkAM4H1sfHGaFoxmurumwCin1Oi6UW1j6Jm/huBJyjyfRJ1lTwDbAH+F1gN7HT3jqhI/H1375No/i5g0sjWOO++CXwe6IrGJ1FE+6NYQsGyTNO5uL0VzT4ys2rgx8BfuPvugYpmmTbm9om7d7r7iYRvyKcAx2QrFv0c0/vEzN4JbHH3J+OTsxQds/ujWEKhEZgVG28ANiZUl6RtNrPpANHPLdH0othHZlZKCIT/cvefRJOLep9kuPtO4CHC8ZY6M0tHs+Lvu3ufRPPHAztGtqZ5dQZwgZmtA+4gdCF9kyLaH8USCkuB+dEZBGXAB4B7Eq5TUu4BPhy9/jChXz0z/Y+iM25OA3ZlulTGCjMz4N+Ble7+9disYt4n9WZWF72uBN5OOMD6IHBxVKzvPsnsq4uBX3vUoT4WuPvV7t7g7nMJnxO/dvcPUkz7I+mDGiM1AOcDLxP6S/8m6fqM0Hu+HdgEtBO+0XyU0N/5APBK9HNiVNYIZ2itBp4HFiZd/zzsj7cQmvbPAc9Ew/lFvk+OB56O9skLwDXR9MOA3wOrgB8B5dH0imh8VTT/sKTfQx73zSLg3mLbH7rNhYiIdCuW7iMREcmBQkFERLopFEREpJtCQUREuikURESkm0JBRES6KRREcmBmJ5rZ+bHxC4brFuxm9hdmVjUc6xI5VLpOQSQHZnY54eK1K/Ow7nXRurcNYZmUu3cOd11E1FKQMcXM5kYP0fm36KExv4pu35Ct7OFm9ksze9LMHjGzo6Pp7zOzF6IHz/wmujXKdcD7zewZM3u/mV1uZt+Oyt9qZjdFD/BZY2ZnWXjA0UozuzW2vZvMbFmfh9l8CpgBPGhmD0bTlpjZ81Edvhpbfo+ZXWdmTwCnm9lXzGxF9ACgf87PHpWik/Ql1Ro0DOdAeH5EB3BiNH4ncFk/ZR8A5kevTyXctwbCLS1mRq/rop+XA9+OLds9TniY0R2E22JcCOwG3kD40vVkrC6Z22ekCDeeOz4aXwdMjl7PAF4D6gn35/818O5ongOXZNZFeL6DxeupQcOhDmopyFi01t2fiV4/SQiKXqLbZ78Z+FH0LIHvEh6wA/AocKuZfYzwAZ6Ln7u7EwJls7s/7+5dwPLY9i8xs6cI9xo6lvAUwL5OBh5y960e7s//X8CZ0bxOwh1eIQRPC3CzmV0E7MuxniIDSg9eRGTUaY297gSydR+VEB6ccmLfGe7+cTM7FfhD4BkzO6DMANvs6rP9LiBtZvOAzwEnu3tT1K1UkWU92e7Pn9Hi0XEEd+8ws1OAcwh387yScJtnkUOiloIUJQ8P11lrZu+DcFttMzshen24uz/h7tcQHq84C2gGag5hk7XAXmCXmU0lPP85I77uJ4CzzGxy9GzxJcDDfVcWtXTGu/t9wF8Qnq8scsjUUpBi9kHgJjP7IlBKOC7wLPBPZjaf8K39gWjaa8BVUVfTl4e6IXd/1syeJnQnrSF0UWV8D/gfM9vk7m8zs6sJ9+834D53/9mBa6QG+JmZVUTlPjPUOolko1NSRUSkm7qPRESkm7qPZMwzsxsJz96Nu8Hdb0miPiKFTN1HIiLSTd1HIiLSTaEgIiLdFAoiItJNoSAiIt3+P9cSgNcLvc/9AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators4_1.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "训练后最优n-estimators为446，接下来调整树的参数：max_depth\n",
    "(粗调，参数的步长为2；下一步是在粗调最佳参数周围，将步长降为1，进行精细调整)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [4, 6, 8]}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = list(range(4,10,2))\n",
    "param_test2_1 = dict(max_depth=max_depth)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "F:\\anaconda\\lib\\site-packages\\sklearn\\model_selection\\_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.57850, std: 0.00506, params: {'max_depth': 4},\n",
       "  mean: -0.58470, std: 0.00562, params: {'max_depth': 6},\n",
       "  mean: -0.61569, std: 0.00350, params: {'max_depth': 8}],\n",
       " {'max_depth': 4},\n",
       " -0.5785042098185154)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=446,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=6,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.5,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=3)\n",
    "gsearch2_1.fit(x_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
